import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.Authenticator;
import java.net.PasswordAuthentication;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.StringTokenizer;

import javax.net.ssl.HttpsURLConnection;


// https://ofmext.epa.gov/AQDMRS/ws/rawDataNotify?user=mkchaudhari%40wpi.edu&pw=bolegazelle45&format=AQCSV&param=42401&bdate=20090205&edate=20140206&state=78&county=010
// https://ofmext.epa.gov/AQDMRS/ws/rawDataNotify?user=mkchaudhari%40wpi.edu&pw=bolegazelle45&format=AQCSV&param=44201&bdate=20090205&edate=20140206&state=78&county=010 &=Submit
// https://ofmext.epa.gov/AQDMRS/ws/rawDataNotify?user=mkchaudhari@wpi.edu&pw=bolegazelle45&format=AQCSV&param=44201&bdate=20090205&edate=20140206&state=78&county=010

public class DataMiningManik
{
	private static final String user = "mkchaudhari@wpi.edu";
	private static final String pass = "bolegazelle45";
	private static String[] format = {"DMCSV", "AQCSV", "AQS"};
	@SuppressWarnings("rawtypes")
	private static HashMap parameterCode = new HashMap();
	private static ArrayList locations = new ArrayList();
	
	// URL constants
	private static final String MAIN_URL	= "https://ofmext.epa.gov/AQDMRS/ws/rawDataNotify?user=" + user + "&pw=" + pass;
	private static final String RESULT_URL 	= "https://ofmext.epa.gov/AQDMRS/ws/retrieve?id=";
	
	protected static final String COUNTY_URL= "https://ofmext.epa.gov/AQDMRS/ws/list?name=county&state=";
	
	// years
	private static final int FIVE_YEARS 	= 50001;
	private static final int TEN_YEARS 		= 100001;
	private static final int FIFTEEN_YEARS 	= 15001;
	private final static String USER_AGENT 		= "Mozilla/5.0";

	private static void init()
	{
		parameterCode = populate(new File("param_final"), ":");
		HashMap stateCode = populate(new File("state_final"), "-");
		
		Iterator it = stateCode.entrySet().iterator();
	    while (it.hasNext()) 
	    {
	        Map.Entry state = (Map.Entry)it.next();
			location s1 = new location((String) state.getKey(), (String) state.getValue());
			System.out.println(state.getValue() + " - found " + s1.getCounty().size() + " counties");
			locations.add(s1);
	    }
	    System.out.println("=== init done. num of locations " + locations.size());
	}
	
	public static void main(String[] args) 
	{
		init();

		String url = MAIN_URL;
		url = url + "&format=" + format[1]; 							// format
		url = url + "&param=" + parameterCode.keySet().toArray()[0]; 	// param
		
		// date
		String edate = getCurrentDate();
		String bdate = String.valueOf(Integer.parseInt(edate) - FIVE_YEARS);
		
		url = url + "&bdate=" + bdate;
		url = url + "&edate=" + edate;

		String baseURL = url;
		// iterate through state		
		for (int i=0; i < locations.size(); i++)
		{
			location loc = (location) locations.get(i);
			url = baseURL + "&state=" + loc.getStateId();
			
			String baseStateUrl = url;
			HashMap county = loc.getCounty();

			//iterate through county
			Iterator iterator = county.keySet().iterator();     
			while (iterator.hasNext()) 
			{  
				String key = iterator.next().toString();  
				String value = county.get(key).toString();     
				url = baseStateUrl + "&county=" + key; // + "&=Submit";
				System.out.println(url);
				String requestId = openConnection(url, null);
				
				System.out.println(requestId);
				break;
			}
			break;
		}

		
		//openConnection(RESULT_URL + "2061542547", null);
	}
	
	protected static String openConnection(String url, File f)
	{
		String result = null;
		InputStream is = null;//http.getInputStream();

		BufferedWriter output = null;
		try 
		{
			URL u = new URL(url);
			System.out.println("Opening connection to " + url);
			HttpsURLConnection http = (HttpsURLConnection) u.openConnection();
		    http.setDoOutput(true);
			Authenticator.setDefault( new MyAuthenticator() );
            http.setAllowUserInteraction(true);
            http.setRequestMethod("GET");
            
            //add request header
            http.setRequestProperty("User-Agent", USER_AGENT);
            
			if (http.getResponseCode() >= 400) 
			{
			    is = http.getErrorStream();
			    Thread.dumpStack();
			}
			else 
			{
			    is = http.getInputStream();
			}
			
            BufferedReader reader = new BufferedReader(new InputStreamReader(is));
            StringBuilder stringBuilder = new StringBuilder();
            String line = null;
            while ((line = reader.readLine()) != null)
            {
                stringBuilder.append(line + "\n");
            }
            
            if ( f != null)
            {
            	output = new BufferedWriter(new FileWriter(f));
                output.write(stringBuilder.toString());
            }
            else
            {
                System.out.println(stringBuilder.toString());
                result = stringBuilder.toString();
            }

		}
		catch (Exception e)
		{
			e.printStackTrace();
		}
		finally 
		{
            try {
            	if (output != null)
            		output.close();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
            
            try {
            	if (is != null)
            		is.close();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		
		return result;
	}

	protected static HashMap populate(File file, String tok) 
	{
		HashMap h = new HashMap();

	    try 
	    {
	        FileReader reader = new FileReader(file);
	        BufferedReader buffReader = new BufferedReader(reader);
	        String s;
	        while((s = buffReader.readLine()) != null)
	        {
	        	StringTokenizer tokenizer = new StringTokenizer(s,tok);
       	        while (tokenizer.hasMoreTokens()) 
       	        {
       	        	Object Obj1 = tokenizer.nextToken().trim();
       	        	//System.out.println(Obj1 + "---");
       	        	
       	        	Object Obj2 = tokenizer.nextToken().trim();
       	        	//System.out.println(Obj2 + "---");
       	        	
       	        	h.put(Obj1,Obj2);
       	        }
	        }	        
	    }
	    catch(Exception e)
	    {
	        e.printStackTrace();
	    }
	    
	    //printHash(h);		
	    return h;
	}
	
	private static String getCurrentDate()
	{
		SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("YYYYMMdd");
		Date today = new Date();
		String date = DATE_FORMAT.format(today);

		return date;
	}
	
	protected static void printHash(HashMap map)
	{
		Iterator iterator = map.keySet().iterator();  
		   
		while (iterator.hasNext()) 
		{  
		   String key = iterator.next().toString();  
		   String value = map.get(key).toString();     
		   System.out.println(key + " " + value);  
		}  
	}

}

